/*
 * #%L
 * Alfresco Repository
 * %%
 * Copyright (C) 2005 - 2016 Alfresco Software Limited
 * %%
 * This file is part of the Alfresco software. 
 * If the software was purchased under a paid Alfresco license, the terms of 
 * the paid license agreement will prevail.  Otherwise, the software is 
 * provided under the following open source license terms:
 * 
 * Alfresco is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * Alfresco is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
 * #L%
 */
package org.alfresco.repo.security.sync;

import java.io.IOException;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.text.DateFormat;
import java.util.*;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import javax.management.*;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationEvent;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.dao.ConcurrencyFailureException;
import org.springframework.extensions.surf.util.AbstractLifecycleBean;
import org.springframework.extensions.surf.util.I18NUtil;

import org.alfresco.model.ContentModel;
import org.alfresco.repo.admin.SysAdminParams;
import org.alfresco.repo.batch.BatchProcessor;
import org.alfresco.repo.batch.BatchProcessor.BatchProcessWorker;
import org.alfresco.repo.dictionary.constraint.NameChecker;
import org.alfresco.repo.lock.JobLockService;
import org.alfresco.repo.lock.LockAcquisitionException;
import org.alfresco.repo.management.subsystems.ActivateableBean;
import org.alfresco.repo.management.subsystems.ChildApplicationContextManager;
import org.alfresco.repo.security.authentication.AuthenticationException;
import org.alfresco.repo.security.authentication.AuthenticationUtil;
import org.alfresco.repo.security.authentication.AuthenticationUtil.RunAsWork;
import org.alfresco.repo.security.authentication.AuthenticatorDeletedEvent;
import org.alfresco.repo.security.authority.UnknownAuthorityException;
import org.alfresco.repo.security.sync.ldap.LDAPUserRegistry;
import org.alfresco.repo.transaction.AlfrescoTransactionSupport;
import org.alfresco.repo.transaction.AlfrescoTransactionSupport.TxnReadState;
import org.alfresco.repo.transaction.RetryingTransactionHelper;
import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
import org.alfresco.service.cmr.attributes.AttributeService;
import org.alfresco.service.cmr.repository.InvalidNodeRefException;
import org.alfresco.service.cmr.security.AuthorityService;
import org.alfresco.service.cmr.security.AuthorityType;
import org.alfresco.service.cmr.security.PersonService;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.service.transaction.TransactionService;
import org.alfresco.util.PropertyCheck;
import org.alfresco.util.PropertyMap;
import org.alfresco.util.TraceableThreadFactory;

/**
 * A <code>ChainingUserRegistrySynchronizer</code> is responsible for synchronizing Alfresco's local user (person) and group (authority) information with the external subsystems in the authentication chain (most typically LDAP directories). When the {@link #synchronize(boolean, boolean)} method is called, it visits each {@link UserRegistry} bean in the 'chain' of application contexts, managed by a {@link ChildApplicationContextManager}, and compares its timestamped user and group information with the local users and groups last retrieved from the same source. Any updates and additions made to those users and groups are applied to the local copies. The ordering of each {@link UserRegistry} in the chain determines its precedence when it comes to user and group name collisions. The {@link JobLockService} is used to ensure that in a cluster, no two nodes actually run a synchronize at the same time.
 * <p>
 * The <code>force</code> argument determines whether a complete or partial set of information is queried from the {@link UserRegistry}. When <code>true</code> then <i>all</i> users and groups are queried. With this complete set of information, the synchronizer is able to identify which users and groups have been deleted, so it will delete users and groups as well as update and create them. Since processing all users and groups may be fairly time consuming, it is recommended this mode is only used by a background scheduled synchronization job. When the argument is <code>false</code> then only those users and groups modified since the most recent modification date of all the objects last queried from the same {@link UserRegistry} are retrieved. In this mode, local users and groups are created and updated, but not deleted (except where a name collision with a lower priority {@link UserRegistry} is detected). This 'differential' mode is much faster, and by default is triggered on
 * subsystem startup and also by {@link #createMissingPerson(String)} when a user is successfully authenticated who doesn't yet have a local person object in Alfresco. This should mean that new users and their group information are pulled over from LDAP servers as and when required.
 * 
 * @author dward
 */
public class ChainingUserRegistrySynchronizer extends AbstractLifecycleBean
        implements UserRegistrySynchronizer,
        ChainingUserRegistrySynchronizerStatus,
        TestableChainingUserRegistrySynchronizer,
        ApplicationEventPublisherAware

{
    /** The logger. */
    private static final Log logger = LogFactory.getLog(ChainingUserRegistrySynchronizer.class);

    /** The name of the lock used to ensure that a synchronize does not run on more than one node at the same time. */
    public static final QName LOCK_QNAME = QName.createQName(NamespaceService.SYSTEM_MODEL_1_0_URI,
            "ChainingUserRegistrySynchronizer");

    /** The time this lock will persist for in the database (now only 2 minutes but refreshed at regular intervals). */
    private static final long LOCK_TTL = 1000 * 60 * 2;

    /** The path in the attribute service below which we persist attributes. */
    public static final String ROOT_ATTRIBUTE_PATH = ".ChainingUserRegistrySynchronizer";

    /** The label under which the last group modification timestamp is stored for each zone. */
    private static final String GROUP_LAST_MODIFIED_ATTRIBUTE = "GROUP";

    /** The label under which the last user modification timestamp is stored for each zone. */
    private static final String PERSON_LAST_MODIFIED_ATTRIBUTE = "PERSON";

    /** The label under which the status is stored for each zone. */
    private static final String STATUS_ATTRIBUTE = "STATUS";

    /** The label under which the status is stored for each zone. */
    private static final String LAST_ERROR_ATTRIBUTE = "LAST_ERROR";

    /** The label under which the status is stored for each zone. */
    private static final String START_TIME_ATTRIBUTE = "START_TIME";

    /** The label under which the status is stored for each zone. */
    private static final String END_TIME_ATTRIBUTE = "END_TIME";

    /** The label under which the status is stored for each zone. */
    private static final String SERVER_ATTRIBUTE = "LAST_RUN_HOST";

    /** The label under which the status is stored for each zone. */
    private static final String SUMMARY_ATTRIBUTE = "SUMMARY";

    /** The manager for the autentication chain to be traversed. */
    private ChildApplicationContextManager applicationContextManager;

    /** The name used to look up a {@link UserRegistry} bean in each child application context. */
    private String sourceBeanName;

    /** The authority service. */
    private AuthorityService authorityService;

    /** The person service. */
    private PersonService personService;

    /** The attribute service. */
    private AttributeService attributeService;

    /** The transaction service. */
    private TransactionService transactionService;

    /** The job lock service. */
    private JobLockService jobLockService;

    /** The application event publisher. */
    private ApplicationEventPublisher applicationEventPublisher;

    /** Should we trigger a differential sync when missing people log in?. */
    private boolean syncWhenMissingPeopleLogIn = true;

    /** Should we trigger a differential sync on startup?. */
    private boolean syncOnStartup = true;

    /** Should we auto create a missing person on log in?. */
    private boolean autoCreatePeopleOnLogin = true;

    /** The number of entries to process before reporting progress. */
    private int loggingInterval = 100;

    /** The number of worker threads. */
    private int workerThreads = 2;

    private MBeanServerConnection mbeanServer;

    /** Allow a full sync to perform deletions? */
    private boolean allowDeletions = true;

    /** Controls whether to query for users and groups that have been deleted in LDAP */
    private boolean syncDelete = true;

    /** Validates person names over cm:filename constraint **/
    private NameChecker nameChecker;

    private SysAdminParams sysAdminParams;

    private String externalUserControl = "";

    private String externalUserControlSubsystemName = "";

    public void init()
    {
        PropertyCheck.mandatory(this, "attributeService", attributeService);
        PropertyCheck.mandatory(this, "authorityService", authorityService);
        PropertyCheck.mandatory(this, "personService", personService);
        PropertyCheck.mandatory(this, "attributeService", attributeService);
        PropertyCheck.mandatory(this, "transactionService", transactionService);
        PropertyCheck.mandatory(this, "jobLockService", jobLockService);
        PropertyCheck.mandatory(this, "applicationEventPublisher", applicationEventPublisher);
        PropertyCheck.mandatory(this, "sysAdminParams", sysAdminParams);
    }

    public void setExternalUserControl(String externalUserControl)
    {
        this.externalUserControl = externalUserControl;
    }

    public void setExternalUserControlSubsystemName(String externalUserControlSubsystemName)
    {
        this.externalUserControlSubsystemName = externalUserControlSubsystemName;
    }

    /**
     * Sets name checker
     */
    public void setNameChecker(NameChecker nameChecker)
    {
        this.nameChecker = nameChecker;
    }

    /**
     * Sets the application context manager.
     * 
     * @param applicationContextManager
     *            the applicationContextManager to set
     */
    public void setApplicationContextManager(ChildApplicationContextManager applicationContextManager)
    {
        this.applicationContextManager = applicationContextManager;
    }

    /**
     * Sets the name used to look up a {@link UserRegistry} bean in each child application context.
     * 
     * @param sourceBeanName
     *            the bean name
     */
    public void setSourceBeanName(String sourceBeanName)
    {
        this.sourceBeanName = sourceBeanName;
    }

    /**
     * Sets the authority service.
     * 
     * @param authorityService
     *            the new authority service
     */
    public void setAuthorityService(AuthorityService authorityService)
    {
        this.authorityService = authorityService;
    }

    /**
     * Sets the person service.
     * 
     * @param personService
     *            the new person service
     */
    public void setPersonService(PersonService personService)
    {
        this.personService = personService;
    }

    /**
     * Sets the attribute service.
     * 
     * @param attributeService
     *            the new attribute service
     */
    public void setAttributeService(AttributeService attributeService)
    {
        this.attributeService = attributeService;
    }

    /**
     * Sets the transaction service.
     * 
     * @param transactionService
     *            the transaction service
     */
    public void setTransactionService(TransactionService transactionService)
    {
        this.transactionService = transactionService;
    }

    /**
     * Sets the job lock service.
     * 
     * @param jobLockService
     *            the job lock service
     */
    public void setJobLockService(JobLockService jobLockService)
    {
        this.jobLockService = jobLockService;
    }

    /* (non-Javadoc)
     * 
     * @see org.springframework.context.ApplicationEventPublisherAware#setApplicationEventPublisher(org.springframework.context .ApplicationEventPublisher) */
    public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher)
    {
        this.applicationEventPublisher = applicationEventPublisher;
    }

    /**
     * Controls whether we auto create a missing person on log in.
     * 
     * @param autoCreatePeopleOnLogin
     *            <code>true</code> if we should auto create a missing person on log in
     */
    public void setAutoCreatePeopleOnLogin(boolean autoCreatePeopleOnLogin)
    {
        this.autoCreatePeopleOnLogin = autoCreatePeopleOnLogin;
    }

    /**
     * Controls whether we trigger a differential sync when missing people log in.
     * 
     * @param syncWhenMissingPeopleLogIn
     *            <code>true</code> if we should trigger a sync when missing people log in
     */
    public void setSyncWhenMissingPeopleLogIn(boolean syncWhenMissingPeopleLogIn)
    {
        this.syncWhenMissingPeopleLogIn = syncWhenMissingPeopleLogIn;
    }

    /**
     * Controls whether we trigger a differential sync when the subsystem starts up.
     * 
     * @param syncOnStartup
     *            <code>true</code> if we should trigger a sync on startup
     */
    public void setSyncOnStartup(boolean syncOnStartup)
    {
        this.syncOnStartup = syncOnStartup;
    }

    /**
     * Sets the number of entries to process before reporting progress.
     * 
     * @param loggingInterval
     *            the number of entries to process before reporting progress or zero to disable progress reporting.
     */
    public void setLoggingInterval(int loggingInterval)
    {
        this.loggingInterval = loggingInterval;
    }

    /**
     * Sets the number of worker threads.
     * 
     * @param workerThreads
     *            the number of worker threads
     */
    public void setWorkerThreads(int workerThreads)
    {
        this.workerThreads = workerThreads;
    }

    /**
     * Controls how deleted users and groups are handled. By default is set to true.
     * 
     * @param allowDeletions
     *            If <b>true</b> the entries are deleted from alfresco. If <b>false</b> then they are unlinked from their LDAP authentication zone but remain within alfresco.
     */
    public void setAllowDeletions(boolean allowDeletions)
    {
        this.allowDeletions = allowDeletions;
    }

    /**
     * Controls whether to query for users and groups that have been deleted in LDAP. For large LDAP directories the delete query is expensive and time consuming, needing to read the entire LDAP directory. By default is set to true.
     * 
     * @param syncDelete
     *            If <b>false</b> then LDAP sync does not even attempt to search for deleted users.
     */
    public void setSyncDelete(boolean syncDelete)
    {
        this.syncDelete = syncDelete;
    }

    @Override
    public SynchronizeDiagnostic testSynchronize(String authenticatorName)
    {
        SynchronizeDiagnosticImpl ret = new SynchronizeDiagnosticImpl();

        Collection<String> instanceIds = this.applicationContextManager.getInstanceIds();

        if (instanceIds.contains(authenticatorName))
        {
            UserRegistry plugin;

            ApplicationContext context = this.applicationContextManager.getApplicationContext(authenticatorName);
            plugin = (UserRegistry) context.getBean(this.sourceBeanName);

            // If the bean is ActivateableBean check whether it is active
            if (plugin instanceof ActivateableBean)
            {
                if (!((ActivateableBean) plugin).isActive())
                {
                    ret.setActive(false);
                }
            }

            long groupLastModifiedMillis = getMostRecentUpdateTime(
                    ChainingUserRegistrySynchronizer.GROUP_LAST_MODIFIED_ATTRIBUTE,
                    authenticatorName, false);

            long personLastModifiedMillis = getMostRecentUpdateTime(
                    ChainingUserRegistrySynchronizer.PERSON_LAST_MODIFIED_ATTRIBUTE,
                    authenticatorName, false);

            Date groupLastModified = groupLastModifiedMillis == -1 ? null : new Date(groupLastModifiedMillis);
            Date personLastModified = personLastModifiedMillis == -1 ? null : new Date(personLastModifiedMillis);

            plugin.initSync(groupLastModified, syncDelete);
            ret.setGroups(plugin.getGroupNames());

            ret.setUsers(plugin.getPersonNames());
            if (groupLastModified != null)
            {
                ret.setGroupLastSynced(groupLastModified);
            }
            else
            { // fake a date to test the group query
                groupLastModified = new Date();
            }
            plugin.getGroups(groupLastModified);
            if (personLastModified != null)
            {
                ret.setPersonLastSynced(personLastModified);
            }
            else
            {
                // fake a date to test the person query
                personLastModified = new Date();
            }
            plugin.getPersons(personLastModified);

            return ret;
        }

        Object params[] = {authenticatorName};
        throw new AuthenticationException("authentication.err.validation.authenticator.notfound", params);
    }

    @Override
    public void synchronize(boolean forceUpdate, boolean isFullSync)
    {
        synchronizeInternal(forceUpdate, isFullSync, true);
    }

    private void synchronizeInternal(boolean forceUpdate, boolean isFullSync, final boolean splitTxns)
    {
        if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
        {

            if (forceUpdate)
            {
                ChainingUserRegistrySynchronizer.logger.debug("Running a full sync.");
            }
            else
            {
                ChainingUserRegistrySynchronizer.logger.debug("Running a differential sync.");
            }
            if (allowDeletions)
            {
                ChainingUserRegistrySynchronizer.logger.debug("deletions are allowed");
            }
            else
            {
                ChainingUserRegistrySynchronizer.logger.debug("deletions are not allowed");
            }
            // Don't proceed with the sync if the repository is read only
            if (this.transactionService.isReadOnly())
            {
                ChainingUserRegistrySynchronizer.logger
                        .warn("Unable to proceed with user registry synchronization. Repository is read only.");
                return;
            }
        }

        // Don't proceed with the sync if the repository is read only
        if (this.transactionService.isReadOnly())
        {
            ChainingUserRegistrySynchronizer.logger
                    .warn("Unable to proceed with user registry synchronization. Repository is read only.");
            return;
        }

        // Create a background executor that will refresh our lock. This means we can request a lock with a relatively
        // small persistence time and not worry about it lasting after server restarts. Note we use an independent
        // executor because this is a compound operation that spans accross multiple batch processors.
        String lockToken = null;
        TraceableThreadFactory threadFactory = new TraceableThreadFactory();
        threadFactory.setNamePrefix("ChainingUserRegistrySynchronizer lock refresh");
        threadFactory.setThreadDaemon(true);
        ScheduledExecutorService lockRefresher = new ScheduledThreadPoolExecutor(1, threadFactory);

        // Let's ensure all exceptions get logged
        try
        {
            // First, try to obtain a lock to ensure we are the only node trying to run this job
            try
            {
                if (splitTxns)
                {
                    // If this is an automated sync on startup or scheduled sync, don't even wait around for the lock.
                    // Assume the sync will be completed on another node.
                    lockToken = this.transactionService.getRetryingTransactionHelper().doInTransaction(
                            new RetryingTransactionCallback<String>() {
                                public String execute() throws Throwable
                                {
                                    return ChainingUserRegistrySynchronizer.this.jobLockService.getLock(
                                            ChainingUserRegistrySynchronizer.LOCK_QNAME,
                                            ChainingUserRegistrySynchronizer.LOCK_TTL, 0, 1);
                                }
                            }, false, splitTxns);
                }
                else
                {
                    // If this is a login-triggered sync, give it a few retries before giving up
                    lockToken = this.jobLockService.getLock(ChainingUserRegistrySynchronizer.LOCK_QNAME,
                            ChainingUserRegistrySynchronizer.LOCK_TTL, 3000, 10);
                }
            }
            catch (LockAcquisitionException e)
            {
                // Don't proceed with the sync if it is running on another node
                ChainingUserRegistrySynchronizer.logger
                        .warn("User registry synchronization already running in another thread. Synchronize aborted");
                return;
            }

            // Schedule the lock refresh to run at regular intervals
            final String token = lockToken;
            lockRefresher.scheduleAtFixedRate(new Runnable() {
                public void run()
                {
                    ChainingUserRegistrySynchronizer.this.transactionService.getRetryingTransactionHelper()
                            .doInTransaction(new RetryingTransactionCallback<Object>() {
                                public Object execute() throws Throwable
                                {
                                    ChainingUserRegistrySynchronizer.this.jobLockService.refreshLock(token,
                                            ChainingUserRegistrySynchronizer.LOCK_QNAME,
                                            ChainingUserRegistrySynchronizer.LOCK_TTL);
                                    return null;
                                }
                            }, false, splitTxns);
                }
            }, ChainingUserRegistrySynchronizer.LOCK_TTL / 2, ChainingUserRegistrySynchronizer.LOCK_TTL / 2,
                    TimeUnit.MILLISECONDS);

            Set<String> visitedZoneIds = new TreeSet<String>();
            Collection<String> instanceIds = this.applicationContextManager.getInstanceIds();

            // Work out the set of all zone IDs in the authentication chain so that we can decide which users / groups
            // need 're-zoning'
            Set<String> allZoneIds = new TreeSet<String>();
            for (String id : instanceIds)
            {
                allZoneIds.add(AuthorityService.ZONE_AUTH_EXT_PREFIX + id);
            }

            // Collect the plugins that we can sync : zoneId, plugin
            Map<String, UserRegistry> plugins = new HashMap<String, UserRegistry>();

            for (String id : instanceIds)
            {
                UserRegistry plugin;
                try
                {
                    ApplicationContext context = this.applicationContextManager.getApplicationContext(id);
                    plugin = (UserRegistry) context.getBean(this.sourceBeanName);
                }
                catch (RuntimeException e)
                {
                    // The bean doesn't exist or this subsystem won't start. The reason would have been logged. Ignore and continue.
                    continue;
                }

                if (!(plugin instanceof ActivateableBean) || ((ActivateableBean) plugin).isActive())
                {
                    // yes this plugin needs to be synced
                    plugins.put(id, plugin);
                }
            }

            /**
             * Sync starts here
             */
            notifySyncStart(plugins.keySet());

            for (String id : instanceIds)
            {
                UserRegistry plugin = plugins.get(id);

                if (plugin != null)
                {
                    // If debug is enabled then dump out the contents of the authentication JMX bean
                    if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                    {
                        mbeanServer = (MBeanServerConnection) getApplicationContext().getBean("alfrescoMBeanServer");
                        try
                        {
                            StringBuilder nameBuff = new StringBuilder(200).append("Alfresco:Type=Configuration,Category=Authentication,id1=managed,id2=").append(
                                    URLDecoder.decode(id, "UTF-8"));
                            ObjectName name = new ObjectName(nameBuff.toString());
                            if (mbeanServer != null && mbeanServer.isRegistered(name))
                            {
                                MBeanInfo info = mbeanServer.getMBeanInfo(name);
                                MBeanAttributeInfo[] attributes = info.getAttributes();
                                ChainingUserRegistrySynchronizer.logger.debug(id + " attributes:");
                                for (MBeanAttributeInfo attribute : attributes)
                                {
                                    Object value = mbeanServer.getAttribute(name, attribute.getName());
                                    ChainingUserRegistrySynchronizer.logger.debug(attribute.getName() + " = " + value);
                                }
                            }
                        }
                        catch (UnsupportedEncodingException e)
                        {
                            if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                            {
                                ChainingUserRegistrySynchronizer.logger
                                        .warn("Exception during logging", e);
                            }
                        }
                        catch (MalformedObjectNameException e)
                        {
                            if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                            {
                                ChainingUserRegistrySynchronizer.logger
                                        .warn("Exception during logging", e);
                            }
                        }
                        catch (InstanceNotFoundException e)
                        {
                            if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                            {
                                ChainingUserRegistrySynchronizer.logger
                                        .warn("Exception during logging", e);
                            }
                        }
                        catch (IntrospectionException e)
                        {
                            if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                            {
                                ChainingUserRegistrySynchronizer.logger
                                        .warn("Exception during logging", e);
                            }
                        }
                        catch (AttributeNotFoundException e)
                        {
                            if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                            {
                                ChainingUserRegistrySynchronizer.logger
                                        .warn("Exception during logging", e);
                            }
                        }
                        catch (ReflectionException e)
                        {
                            if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                            {
                                ChainingUserRegistrySynchronizer.logger
                                        .warn("Exception during logging", e);
                            }
                        }
                        catch (MBeanException e)
                        {
                            if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                            {
                                ChainingUserRegistrySynchronizer.logger
                                        .warn("Exception during logging", e);
                            }
                        }
                        catch (IOException e)
                        {
                            if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                            {
                                ChainingUserRegistrySynchronizer.logger
                                        .warn("Exception during logging", e);
                            }
                        }
                    } // end of debug dump of active JMX bean
                    if (ChainingUserRegistrySynchronizer.logger.isInfoEnabled())
                    {
                        ChainingUserRegistrySynchronizer.logger
                                .info("Synchronizing users and groups with user registry '" + id + "'");
                    }
                    if (isFullSync && ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                    {
                        if (allowDeletions)
                        {
                            ChainingUserRegistrySynchronizer.logger
                                    .warn("Some users and groups previously created by synchronization with this user registry may be removed.");
                        }
                        else
                        {
                            ChainingUserRegistrySynchronizer.logger
                                    .warn("Deletions are disabled. Users and groups removed from this registry will be logged only and will remain in the repository. Users previously found in a different registry will be moved in the repository rather than recreated.");
                        }
                    }
                    // Work out whether we should do the work in a separate transaction (it's most performant if we
                    // bunch it into small transactions, but if we are doing a sync on login, it has to be the same
                    // transaction)
                    boolean requiresNew = splitTxns
                            || AlfrescoTransactionSupport.getTransactionReadState() == TxnReadState.TXN_READ_ONLY;

                    try
                    {
                        /**
                         * Do the sync with the specified plugin
                         */
                        syncWithPlugin(id, plugin, forceUpdate, isFullSync, requiresNew, visitedZoneIds, allZoneIds);

                        this.applicationEventPublisher.publishEvent(new SynchronizeDirectoryEndEvent(this, id));
                    }
                    catch (final RuntimeException e)
                    {
                        notifySyncDirectoryEnd(id, e);
                        throw e;
                    }
                } // if plugin exists
            } // for each instanceId

            // End of successful synchronization here
            notifySyncEnd();
        }
        catch (final RuntimeException e)
        {
            notifySyncEnd(e);
            ChainingUserRegistrySynchronizer.logger.error("Synchronization aborted due to error", e);
            throw e;
        }
        finally
        {
            // Release the lock if necessary
            if (lockToken != null)
            {
                // Cancel the lock refresher
                // Because we may hit a perfect storm when trying to interrupt workers in their unsynchronized getTask()
                // method we can't wait indefinitely and may have to retry the shutdown
                int trys = 0;
                do
                {
                    lockRefresher.shutdown();
                    try
                    {
                        lockRefresher.awaitTermination(ChainingUserRegistrySynchronizer.LOCK_TTL, TimeUnit.MILLISECONDS);
                    }
                    catch (InterruptedException e)
                    {}
                } while (!lockRefresher.isTerminated() && trys++ < 3);
                if (!lockRefresher.isTerminated())
                {
                    lockRefresher.shutdownNow();
                    ChainingUserRegistrySynchronizer.logger.error("Failed to shut down lock refresher");
                }

                final String token = lockToken;
                this.transactionService.getRetryingTransactionHelper().doInTransaction(
                        new RetryingTransactionCallback<Object>() {
                            public Object execute() throws Throwable
                            {
                                ChainingUserRegistrySynchronizer.this.jobLockService.releaseLock(token,
                                        ChainingUserRegistrySynchronizer.LOCK_QNAME);
                                return null;
                            }
                        }, false, splitTxns);
            }
        }
    }

    /* (non-Javadoc)
     * 
     * @see org.alfresco.repo.security.sync.UserRegistrySynchronizer#getPersonMappedProperties(java.lang.String) */
    public Set<QName> getPersonMappedProperties(String username)
    {
        Set<String> authorityZones = this.authorityService.getAuthorityZones(username);
        if (authorityZones == null)
        {
            return Collections.emptySet();
        }
        Collection<String> instanceIds = this.applicationContextManager.getInstanceIds();

        // Visit the user registries in priority order and return the person mapping of the first registry that matches
        // one of the person's zones
        for (String id : instanceIds)
        {
            String zoneId = AuthorityService.ZONE_AUTH_EXT_PREFIX + id;
            if (!authorityZones.contains(zoneId))
            {
                continue;
            }
            try
            {
                ApplicationContext context = this.applicationContextManager.getApplicationContext(id);
                UserRegistry plugin = (UserRegistry) context.getBean(this.sourceBeanName);
                if (!(plugin instanceof ActivateableBean) || ((ActivateableBean) plugin).isActive())
                {
                    return plugin.getPersonMappedProperties();
                }
            }
            catch (RuntimeException e)
            {
                // The bean doesn't exist or this subsystem won't start. The reason would have been logged. Ignore and continue.
            }
        }

        return Collections.emptySet();
    }

    @Override
    public boolean createMissingPerson(String userName)
    {
        // synchronise or auto-create the missing person if we are allowed
        if (userName != null && !userName.equals(AuthenticationUtil.getSystemUserName()))
        {
            if (this.syncWhenMissingPeopleLogIn)
            {
                try
                {
                    synchronizeInternal(false, false, false);
                }
                catch (Exception e)
                {
                    // We don't want to fail the whole login if we can help it
                    ChainingUserRegistrySynchronizer.logger.warn("User authenticated but failed to sync with user registry", e);
                }
                if (this.personService.personExists(userName))
                {
                    return true;
                }
            }
            if (this.autoCreatePeopleOnLogin && this.personService.createMissingPeople())
            {
                AuthorityType authorityType = AuthorityType.getAuthorityType(userName);
                if (authorityType == AuthorityType.USER)
                {
                    this.personService.getPerson(userName);
                    return true;
                }
            }
        }
        return false;
    }

    /**
     * Lookup table for sync process used by syncWithPlugin
     * 
     */
    private enum SyncProcess
    {
        GROUP_ANALYSIS("1 Group Analysis"), MISSING_AUTHORITY("2 Missing Authority Scanning"), GROUP_CREATION_AND_ASSOCIATION_DELETION("3 Group Creation and Association Deletion"), GROUP_ASSOCIATION_CREATION("4 Group Association Creation"), PERSON_ASSOCIATION("5 User Association"), USER_CREATION("6 User Creation and Association"), AUTHORITY_DELETION("7 Authority Deletion");

        SyncProcess(String title)
        {
            this.title = title;
        }

        public String getTitle(String zone)
        {
            return "Synchronization,Category=directory,id1=" + zone + ",id2=" + title;
        }

        private String title;
    }

    /**
     * Synchronizes local groups and users with a {@link UserRegistry} for a particular zone, optionally handling deletions.
     * 
     * @param zone
     *            the zone id. This identifier is used to tag all created groups and users, so that in the future we can tell those that have been deleted from the registry.
     * @param userRegistry
     *            the user registry for the zone.
     * @param forceUpdate
     *            Should the complete set of users and groups be updated / created locally or just those known to have changed since the last sync? When <code>true</code> then <i>all</i> users and groups are queried from the user registry and updated locally. When <code>false</code> then each source is only queried for those users and groups modified since the most recent modification date of all the objects last queried from that same source.
     * @param isFullSync
     *            Should a complete set of user and group IDs be queried from the user registries in order to determine deletions? This parameter is independent of <code>force</code> as a separate query is run to process updates.
     * @param splitTxns
     *            Can the modifications to Alfresco be split across multiple transactions for maximum performance? If <code>true</code>, users and groups are created/updated in batches for increased performance. If <code>false</code>, all users and groups are processed in the current transaction. This is required if calling synchronously (e.g. in response to an authentication event in the same transaction).
     * @param visitedZoneIds
     *            the set of zone ids already processed. These zones have precedence over the current zone when it comes to group name 'collisions'. If a user or group is queried that already exists locally but is tagged with one of the zones in this set, then it will be ignored as this zone has lower priority.
     * @param allZoneIds
     *            the set of all zone ids in the authentication chain. Helps us work out whether the zone information recorded against a user or group is invalid for the current authentication chain and whether the user or group needs to be 're-zoned'.
     */
    private void syncWithPlugin(final String zone, UserRegistry userRegistry, boolean forceUpdate,
            boolean isFullSync, boolean splitTxns, final Set<String> visitedZoneIds, final Set<String> allZoneIds)
    {
        // Create a prefixed zone ID for use with the authority service
        final String zoneId = AuthorityService.ZONE_AUTH_EXT_PREFIX + zone;

        // Batch Process Names
        final String reservedBatchProcessNames[] = {
                SyncProcess.GROUP_ANALYSIS.getTitle(zone),
                SyncProcess.USER_CREATION.getTitle(zone),
                SyncProcess.MISSING_AUTHORITY.getTitle(zone),
                SyncProcess.GROUP_CREATION_AND_ASSOCIATION_DELETION.getTitle(zone),
                SyncProcess.GROUP_ASSOCIATION_CREATION.getTitle(zone),
                SyncProcess.PERSON_ASSOCIATION.getTitle(zone),
                SyncProcess.AUTHORITY_DELETION.getTitle(zone)
        };

        notifySyncDirectoryStart(zone, reservedBatchProcessNames);

        // Ensure that the zoneId exists before multiple threads start using it
        this.transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Void>() {
            @Override
            public Void execute() throws Throwable
            {
                authorityService.getOrCreateZone(zoneId);
                return null;
            }
        }, false, splitTxns);

        // The set of zones we associate with new objects (default plus registry specific)
        final Set<String> zoneSet = getZones(zoneId);

        long lastModifiedMillis = forceUpdate ? -1
                : getMostRecentUpdateTime(
                        ChainingUserRegistrySynchronizer.GROUP_LAST_MODIFIED_ATTRIBUTE, zoneId, splitTxns);
        Date lastModified = lastModifiedMillis == -1 ? null : new Date(lastModifiedMillis);
        userRegistry.initSync(lastModified, syncDelete);
        if (ChainingUserRegistrySynchronizer.logger.isInfoEnabled())
        {
            if (lastModified == null)
            {
                ChainingUserRegistrySynchronizer.logger.info("Retrieving all groups from user registry '" + zone + "'");
            }
            else
            {
                ChainingUserRegistrySynchronizer.logger.info("Retrieving groups changed since "
                        + DateFormat.getDateTimeInstance().format(lastModified) + " from user registry '" + zone + "'");
            }
        }

        // First, analyze the group structure. Create maps of authorities to their parents for associations to create
        // and delete. Also deal with 'overlaps' with other zones in the authentication chain.
        final BatchProcessor<NodeDescription> groupProcessor = new BatchProcessor<NodeDescription>(
                SyncProcess.GROUP_ANALYSIS.getTitle(zone),
                this.transactionService.getRetryingTransactionHelper(),
                userRegistry.getGroups(lastModified),
                this.workerThreads,
                20,
                this.applicationEventPublisher,
                ChainingUserRegistrySynchronizer.logger,
                this.loggingInterval);
        class Analyzer extends BaseBatchProcessWorker<NodeDescription>
        {
            private final Map<String, NodeDescription> nodeDescriptions = new HashMap<>();
            private final Map<String, String> groupsToCreate = new TreeMap<String, String>();
            private final Map<String, Set<String>> personParentAssocsToCreate = newPersonMap();
            private final Map<String, Set<String>> personParentAssocsToDelete = newPersonMap();
            private final List<String> personToRezone = new LinkedList<>();
            private Map<String, Set<String>> groupParentAssocsToCreate = new TreeMap<String, Set<String>>();
            private final Map<String, Set<String>> groupParentAssocsToDelete = new TreeMap<String, Set<String>>();
            private final List<String> groupToRezone = new LinkedList<>();
            private final Map<String, Set<String>> finalGroupChildAssocs = new TreeMap<String, Set<String>>();
            private List<String> personsProcessed = new LinkedList<String>();
            private Set<String> allZonePersons = Collections.emptySet();
            private Set<String> deletionCandidates;

            private long latestTime;

            public Analyzer(final long latestTime)
            {
                this.latestTime = latestTime;
            }

            public long getLatestTime()
            {
                return this.latestTime;
            }

            public Set<String> getDeletionCandidates()
            {
                return this.deletionCandidates;
            }

            public String getIdentifier(NodeDescription entry)
            {
                return entry.getSourceId();
            }

            public void process(NodeDescription group) throws Throwable
            {
                PropertyMap groupProperties = group.getProperties();
                String groupName = (String) groupProperties.get(ContentModel.PROP_AUTHORITY_NAME);
                String groupShortName = ChainingUserRegistrySynchronizer.this.authorityService.getShortName(groupName);
                Set<String> groupZones = ChainingUserRegistrySynchronizer.this.authorityService
                        .getAuthorityZones(groupName);

                if (groupZones == null)
                {
                    // The group did not exist at all
                    updateGroup(group, false);
                }
                else
                {
                    // Check whether the group is in any of the authentication chain zones
                    Set<String> intersection = new TreeSet<String>(groupZones);
                    intersection.retainAll(allZoneIds);
                    // Check whether the group is in any of the higher priority authentication chain zones
                    Set<String> visited = new TreeSet<String>(intersection);
                    visited.retainAll(visitedZoneIds);

                    if (groupZones.contains(zoneId))
                    {
                        // The group already existed in this zone: update the group
                        updateGroup(group, true);
                    }
                    else if (!visited.isEmpty())
                    {
                        // A group that exists in a different zone with higher precedence
                        return;
                    }
                    else if (!allowDeletions || intersection.isEmpty())
                    {
                        // Deletions are disallowed or the group exists, but not in a zone that's in the authentication
                        // chain. May be due to upgrade or zone changes. Let's re-zone them
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                        {
                            ChainingUserRegistrySynchronizer.logger.warn("Updating group '" + groupShortName
                                    + "'. This group will in future be assumed to originate from user registry '"
                                    + zone + "'.");
                        }
                        updateAuthorityZones(groupName, groupZones, zoneSet);

                        // The group now exists in this zone: update the group
                        updateGroup(group, true);
                    }
                    else
                    {
                        // The group existed, but in a zone with lower precedence
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                        {
                            ChainingUserRegistrySynchronizer.logger
                                    .warn("Recreating occluded group '"
                                            + groupShortName
                                            + "'. This group was previously created through synchronization with a lower priority user registry.");
                        }
                        ChainingUserRegistrySynchronizer.this.authorityService.deleteAuthority(groupName);

                        // create the group
                        updateGroup(group, false);
                    }
                }

                synchronized (this)
                {
                    // Maintain the last modified date
                    Date groupLastModified = group.getLastModified();
                    if (groupLastModified != null)
                    {
                        this.latestTime = Math.max(this.latestTime, groupLastModified.getTime());
                    }
                }
            }

            // Recursively walks and caches the authorities relating to and from this group so that we can later detect potential cycles
            private Set<String> getContainedAuthorities(String groupName)
            {
                // Return the cached children if it is processed
                Set<String> children = this.finalGroupChildAssocs.get(groupName);
                if (children != null)
                {
                    return children;
                }

                // First, recurse to the parent most authorities
                for (String parent : ChainingUserRegistrySynchronizer.this.authorityService.getContainingAuthorities(
                        null, groupName, true))
                {
                    getContainedAuthorities(parent);
                }

                // Now descend on unprocessed parents.
                return cacheContainedAuthorities(groupName);
            }

            private Set<String> cacheContainedAuthorities(String groupName)
            {
                // Return the cached children if it is processed
                Set<String> children = this.finalGroupChildAssocs.get(groupName);
                if (children != null)
                {
                    return children;
                }

                // Descend on unprocessed parents.
                children = ChainingUserRegistrySynchronizer.this.authorityService.getContainedAuthorities(null,
                        groupName, true);
                this.finalGroupChildAssocs.put(groupName, children);

                for (String child : children)
                {
                    if (AuthorityType.getAuthorityType(child) != AuthorityType.USER)
                    {
                        cacheContainedAuthorities(child);
                    }
                }
                return children;
            }

            private synchronized void updateGroup(NodeDescription group, boolean existed)
            {
                PropertyMap groupProperties = group.getProperties();
                String groupName = (String) groupProperties.get(ContentModel.PROP_AUTHORITY_NAME);
                nodeDescriptions.put(groupName, group);
                String groupDisplayName = (String) groupProperties.get(ContentModel.PROP_AUTHORITY_DISPLAY_NAME);
                if (groupDisplayName == null)
                {
                    groupDisplayName = ChainingUserRegistrySynchronizer.this.authorityService.getShortName(groupName);
                }

                // Divide the child associations into person and group associations, dealing with case sensitivity
                Set<String> newChildPersons = newPersonSet();
                Set<String> newChildGroups = new TreeSet<String>();

                for (String child : group.getChildAssociations())
                {
                    if (AuthorityType.getAuthorityType(child) == AuthorityType.USER)
                    {
                        newChildPersons.add(child);
                    }
                    else
                    {
                        newChildGroups.add(child);
                    }
                }

                // Account for differences if already existing
                if (existed)
                {
                    // Update the display name now
                    ChainingUserRegistrySynchronizer.this.authorityService.setAuthorityDisplayName(groupName,
                            groupDisplayName);

                    // Work out the association differences
                    for (String child : new TreeSet<String>(getContainedAuthorities(groupName)))
                    {
                        if (AuthorityType.getAuthorityType(child) == AuthorityType.USER)
                        {
                            if (!newChildPersons.remove(child))
                            {
                                recordParentAssociationDeletion(child, groupName);
                            }
                        }
                        else
                        {
                            if (!newChildGroups.remove(child))
                            {
                                recordParentAssociationDeletion(child, groupName);
                            }
                        }
                    }
                }
                // Mark as created if new
                else
                {
                    // Make sure each group to be created features in the association deletion map (as these are handled in the same phase)
                    recordParentAssociationDeletion(groupName, null);
                    this.groupsToCreate.put(groupName, groupDisplayName);
                }

                // Create new associations
                for (String child : newChildPersons)
                {
                    // Make sure each person with association changes features as a key in the deletion map
                    recordParentAssociationDeletion(child, null);
                    recordParentAssociationCreation(child, groupName);
                }
                for (String child : newChildGroups)
                {
                    // Make sure each group with association changes features as a key in the deletion map
                    recordParentAssociationDeletion(child, null);
                    recordParentAssociationCreation(child, groupName);
                }
            }

            private void recordParentAssociationDeletion(String child, String parent)
            {
                Map<String, Set<String>> parentAssocs;
                if (AuthorityType.getAuthorityType(child) == AuthorityType.USER)
                {
                    parentAssocs = this.personParentAssocsToDelete;
                }
                else
                {
                    // Reflect the change in the map of final group associations (for cycle detection later)
                    parentAssocs = this.groupParentAssocsToDelete;
                    if (parent != null)
                    {
                        Set<String> children = this.finalGroupChildAssocs.get(parent);
                        children.remove(child);
                    }
                }
                Set<String> parents = parentAssocs.get(child);
                if (parents == null)
                {
                    parents = new TreeSet<String>();
                    parentAssocs.put(child, parents);
                }
                if (parent != null)
                {
                    parents.add(parent);
                }
            }

            private void recordParentAssociationCreation(String child, String parent)
            {
                Map<String, Set<String>> parentAssocs = AuthorityType.getAuthorityType(child) == AuthorityType.USER ? this.personParentAssocsToCreate : this.groupParentAssocsToCreate;
                Set<String> parents = parentAssocs.get(child);
                if (parents == null)
                {
                    parents = new TreeSet<String>();
                    parentAssocs.put(child, parents);
                }
                if (parent != null)
                {
                    parents.add(parent);
                }
            }

            private void recordParentAssociationAuthoritiesToRezone(String child)
            {
                if (child != null)
                {
                    List<String> toRezone = AuthorityType.getAuthorityType(child) == AuthorityType.USER
                            ? this.personToRezone
                            : this.groupToRezone;
                    toRezone.add(child);
                }
            }

            private void validateGroupParentAssocsToCreate()
            {
                Iterator<Map.Entry<String, Set<String>>> i = this.groupParentAssocsToCreate.entrySet().iterator();
                while (i.hasNext())
                {
                    Map.Entry<String, Set<String>> entry = i.next();
                    String group = entry.getKey();
                    Set<String> parents = entry.getValue();
                    Deque<String> visited = new LinkedList<String>();
                    Iterator<String> j = parents.iterator();
                    while (j.hasNext())
                    {
                        String parent = j.next();
                        visited.add(parent);
                        if (validateAuthorityChildren(visited, group))
                        {
                            // The association validated - commit it
                            Set<String> children = finalGroupChildAssocs.get(parent);
                            if (children == null)
                            {
                                children = new TreeSet<String>();
                                finalGroupChildAssocs.put(parent, children);
                            }
                            children.add(group);
                        }
                        else
                        {
                            // The association did not validate - prune it out
                            if (logger.isWarnEnabled())
                            {
                                ChainingUserRegistrySynchronizer.logger.warn("Not adding group '"
                                        + ChainingUserRegistrySynchronizer.this.authorityService.getShortName(group)
                                        + "' to group '"
                                        + ChainingUserRegistrySynchronizer.this.authorityService.getShortName(parent)
                                        + "' as this creates a cyclic relationship");
                            }
                            j.remove();
                        }
                        visited.removeLast();
                    }
                    if (parents.isEmpty())
                    {
                        i.remove();
                    }
                }

                // Sort the group associations in parent-first order (root groups first) to minimize reindexing overhead
                Map<String, Set<String>> sortedGroupAssociations = new LinkedHashMap<String, Set<String>>(
                        this.groupParentAssocsToCreate.size() * 2);
                Deque<String> visited = new LinkedList<String>();
                for (String authority : this.groupParentAssocsToCreate.keySet())
                {
                    visitGroupParentAssocs(visited, authority, this.groupParentAssocsToCreate, sortedGroupAssociations);
                }

                this.groupParentAssocsToCreate = sortedGroupAssociations;
            }

            private boolean validateAuthorityChildren(Deque<String> visited, String authority)
            {
                if (AuthorityType.getAuthorityType(authority) == AuthorityType.USER)
                {
                    return true;
                }
                if (visited.contains(authority))
                {
                    return false;
                }
                visited.add(authority);
                try
                {
                    Set<String> children = this.finalGroupChildAssocs.get(authority);
                    if (children != null)
                    {
                        for (String child : children)
                        {
                            if (!validateAuthorityChildren(visited, child))
                            {
                                return false;
                            }
                        }
                    }
                    return true;
                }
                finally
                {
                    visited.removeLast();
                }
            }

            /**
             * Visits the given authority by recursively visiting its parents in associationsOld and then adding the authority to associationsNew. Used to sort associationsOld into 'parent-first' order to minimize reindexing overhead.
             * 
             * @param visited
             *            The ancestors that form the path to the authority to visit. Allows detection of cyclic child associations.
             * @param authority
             *            the authority to visit
             * @param associationsOld
             *            the association map to sort
             * @param associationsNew
             *            the association map to add to in parent-first order
             */
            private boolean visitGroupParentAssocs(Deque<String> visited, String authority,
                    Map<String, Set<String>> associationsOld, Map<String, Set<String>> associationsNew)
            {
                if (visited.contains(authority))
                {
                    // Prevent cyclic paths (Shouldn't happen as we've already validated)
                    return false;
                }
                visited.add(authority);
                try
                {
                    if (!associationsNew.containsKey(authority))
                    {
                        Set<String> oldParents = associationsOld.get(authority);
                        if (oldParents != null)
                        {
                            Set<String> newParents = new TreeSet<String>();

                            for (String parent : oldParents)
                            {
                                if (visitGroupParentAssocs(visited, parent, associationsOld, associationsNew))
                                {
                                    newParents.add(parent);
                                }
                            }
                            associationsNew.put(authority, newParents);
                        }
                    }
                    return true;
                }
                finally
                {
                    visited.removeLast();
                }
            }

            private Set<String> newPersonSet()
            {
                return ChainingUserRegistrySynchronizer.this.personService.getUserNamesAreCaseSensitive() ? new TreeSet<String>()
                        : new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
            }

            private Map<String, Set<String>> newPersonMap()
            {
                return ChainingUserRegistrySynchronizer.this.personService.getUserNamesAreCaseSensitive() ? new TreeMap<String, Set<String>>()
                        : new TreeMap<String, Set<String>>(String.CASE_INSENSITIVE_ORDER);
            }

            private void logRetainParentAssociations(Map<String, Set<String>> parentAssocs, Set<String> toRetain)
            {
                Iterator<Map.Entry<String, Set<String>>> i = parentAssocs.entrySet().iterator();
                StringBuilder groupList = null;
                while (i.hasNext())
                {
                    Map.Entry<String, Set<String>> entry = i.next();
                    String child = entry.getKey();
                    if (!toRetain.contains(child))
                    {
                        if (!shouldRezone(child))
                        {
                            if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                            {
                                if (groupList == null)
                                {
                                    groupList = new StringBuilder(1024);
                                }
                                else
                                {
                                    groupList.setLength(0);
                                }
                                for (String parent : entry.getValue())
                                {
                                    if (groupList.length() > 0)
                                    {
                                        groupList.append(", ");
                                    }
                                    groupList.append('\'').append(
                                            ChainingUserRegistrySynchronizer.this.authorityService.getShortName(parent))
                                            .append('\'');

                                }
                                ChainingUserRegistrySynchronizer.logger.debug("Ignoring non-existent member '"
                                        + ChainingUserRegistrySynchronizer.this.authorityService.getShortName(child)
                                        + "' in groups {" + groupList.toString() + "}");
                            }
                            i.remove();
                        }
                        else
                        {
                            recordParentAssociationAuthoritiesToRezone(child);
                        }
                    }
                }
            }

            private boolean shouldRezone(String authorityName)
            {
                boolean exists = authorityService.authorityExists(authorityName);

                if (exists)
                {
                    Set<String> zones = ChainingUserRegistrySynchronizer.this.authorityService.getAuthorityZones(authorityName);
                    return isInZone(authorityName, zones, AuthorityService.ZONE_AUTH_ALFRESCO) && !isInZone(authorityName, zones, zoneId);
                }

                return false;
            }

            private void processGroups(UserRegistry userRegistry, boolean isFullSync, boolean splitTxns)
            {
                // MNT-12454 fix. If syncDelete is false, there is no need to pull all users and all groups from LDAP during the full synchronization.
                if ((syncDelete || !groupsToCreate.isEmpty()) && (isFullSync || !this.groupParentAssocsToDelete.isEmpty()))
                {
                    final Set<String> allZonePersons = newPersonSet();
                    final Set<String> allZoneGroups = new TreeSet<String>();

                    // Add in current set of known authorities
                    ChainingUserRegistrySynchronizer.this.transactionService.getRetryingTransactionHelper()
                            .doInTransaction(new RetryingTransactionCallback<Void>() {
                                public Void execute() throws Throwable
                                {
                                    allZonePersons.addAll(ChainingUserRegistrySynchronizer.this.authorityService
                                            .getAllAuthoritiesInZone(zoneId, AuthorityType.USER));
                                    allZoneGroups.addAll(ChainingUserRegistrySynchronizer.this.authorityService
                                            .getAllAuthoritiesInZone(zoneId, AuthorityType.GROUP));
                                    return null;
                                }
                            }, true, splitTxns);

                    allZoneGroups.addAll(this.groupsToCreate.keySet());

                    // Prune our set of authorities according to deletions
                    if (isFullSync)
                    {
                        final Set<String> personDeletionCandidates = newPersonSet();
                        personDeletionCandidates.addAll(allZonePersons);

                        final Set<String> groupDeletionCandidates = new TreeSet<String>();
                        groupDeletionCandidates.addAll(allZoneGroups);

                        this.deletionCandidates = new TreeSet<String>();

                        for (String person : userRegistry.getPersonNames())
                        {
                            personDeletionCandidates.remove(person);
                        }

                        for (String group : userRegistry.getGroupNames())
                        {
                            groupDeletionCandidates.remove(group);
                        }

                        this.deletionCandidates = new TreeSet<String>();
                        this.deletionCandidates.addAll(personDeletionCandidates);
                        this.deletionCandidates.addAll(groupDeletionCandidates);

                        if (allowDeletions)
                        {
                            allZonePersons.removeAll(personDeletionCandidates);
                            allZoneGroups.removeAll(groupDeletionCandidates);
                        }
                        else
                        {
                            // Complete association deletion information by scanning deleted groups
                            BatchProcessor<String> groupScanner = new BatchProcessor<String>(zone
                                    + " Missing Authority Scanning",
                                    ChainingUserRegistrySynchronizer.this.transactionService.getRetryingTransactionHelper(),
                                    this.deletionCandidates,
                                    ChainingUserRegistrySynchronizer.this.workerThreads, 20,
                                    ChainingUserRegistrySynchronizer.this.applicationEventPublisher,
                                    ChainingUserRegistrySynchronizer.logger,
                                    ChainingUserRegistrySynchronizer.this.loggingInterval);
                            groupScanner.process(new BaseBatchProcessWorker<String>() {

                                @Override
                                public String getIdentifier(String entry)
                                {
                                    return entry;
                                }

                                @Override
                                public void process(String authority) throws Throwable
                                {
                                    // MNT-12454 fix. Modifies an authority's zone. Move authority from AUTH.EXT.LDAP1 to AUTH.ALF.
                                    updateAuthorityZones(authority, Collections.singleton(zoneId),
                                            Collections.singleton(AuthorityService.ZONE_AUTH_ALFRESCO));
                                }
                            }, splitTxns);
                        }

                    }

                    // Prune the group associations now that we have complete information
                    this.groupParentAssocsToCreate.keySet().retainAll(allZoneGroups);
                    logRetainParentAssociations(this.groupParentAssocsToCreate, allZoneGroups);
                    this.finalGroupChildAssocs.keySet().retainAll(allZoneGroups);

                    // Pruning person associations will have to wait until we have passed over all persons and built up
                    // this set
                    this.allZonePersons = allZonePersons;

                    if (!this.groupParentAssocsToDelete.isEmpty())
                    {
                        // Create/update the groups and delete parent associations to be deleted
                        // Batch 4 Group Creation and Association Deletion
                        BatchProcessor<Map.Entry<String, Set<String>>> groupCreator = new BatchProcessor<Map.Entry<String, Set<String>>>(
                                SyncProcess.GROUP_CREATION_AND_ASSOCIATION_DELETION.getTitle(zone),
                                ChainingUserRegistrySynchronizer.this.transactionService.getRetryingTransactionHelper(),
                                this.groupParentAssocsToDelete.entrySet(),
                                ChainingUserRegistrySynchronizer.this.workerThreads, 20,
                                ChainingUserRegistrySynchronizer.this.applicationEventPublisher,
                                ChainingUserRegistrySynchronizer.logger,
                                ChainingUserRegistrySynchronizer.this.loggingInterval);
                        groupCreator.process(new BaseBatchProcessWorker<Map.Entry<String, Set<String>>>() {
                            public String getIdentifier(Map.Entry<String, Set<String>> entry)
                            {
                                return entry.getKey() + " " + entry.getValue();
                            }

                            public void process(Map.Entry<String, Set<String>> entry) throws Throwable
                            {
                                String child = entry.getKey();

                                String groupDisplayName = Analyzer.this.groupsToCreate.get(child);
                                if (groupDisplayName != null)
                                {
                                    String groupShortName = ChainingUserRegistrySynchronizer.this.authorityService
                                            .getShortName(child);
                                    if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                                    {
                                        ChainingUserRegistrySynchronizer.logger.debug("Creating group '"
                                                + groupShortName + "'");
                                    }
                                    // create the group
                                    Map<QName, Serializable> groupProperties = Optional.ofNullable(Analyzer.this.nodeDescriptions.get(child))
                                            .map(NodeDescription::getProperties)
                                            .orElse(new PropertyMap());
                                    ChainingUserRegistrySynchronizer.this.authorityService.createAuthority(
                                            AuthorityType.getAuthorityType(child), groupShortName, groupDisplayName, zoneSet, groupProperties);
                                }
                                else
                                {
                                    // Maintain association deletions now. The creations will have to be done later once
                                    // we have performed all the deletions in order to avoid creating cycles
                                    maintainAssociationDeletions(child);
                                }
                            }
                        }, splitTxns);
                    }
                }
            }

            private void finalizeAssociations(UserRegistry userRegistry, boolean splitTxns)
            {
                // First validate the group associations to be created for potential cycles. Remove any offending association
                validateGroupParentAssocsToCreate();

                // Now go ahead and create the group associations
                if (!this.groupParentAssocsToCreate.isEmpty())
                {
                    // Batch 5 Group Association Creation
                    BatchProcessor<Map.Entry<String, Set<String>>> groupCreator = new BatchProcessor<Map.Entry<String, Set<String>>>(
                            SyncProcess.GROUP_ASSOCIATION_CREATION.getTitle(zone),
                            ChainingUserRegistrySynchronizer.this.transactionService.getRetryingTransactionHelper(),
                            this.groupParentAssocsToCreate.entrySet(),
                            ChainingUserRegistrySynchronizer.this.workerThreads, 20,
                            ChainingUserRegistrySynchronizer.this.applicationEventPublisher,
                            ChainingUserRegistrySynchronizer.logger,
                            ChainingUserRegistrySynchronizer.this.loggingInterval);
                    groupCreator.process(new BaseBatchProcessWorker<Map.Entry<String, Set<String>>>() {
                        public String getIdentifier(Map.Entry<String, Set<String>> entry)
                        {
                            return entry.getKey() + " " + entry.getValue();
                        }

                        public void process(Map.Entry<String, Set<String>> entry) throws Throwable
                        {
                            maintainAssociationCreations(entry.getKey());
                            maintainAssociationCreationsToRezone(entry.getKey());
                        }
                    }, splitTxns);
                }

                // Remove all the associations we have already dealt with
                this.personParentAssocsToDelete.keySet().removeAll(this.personsProcessed);

                // Filter out associations to authorities that simply can't exist (and log if debugging is enabled)
                logRetainParentAssociations(this.personParentAssocsToCreate, this.allZonePersons);

                // Update associations to persons not updated themselves
                if (!this.personParentAssocsToDelete.isEmpty())
                {
                    // Batch 6 Person Association
                    BatchProcessor<Map.Entry<String, Set<String>>> groupCreator = new BatchProcessor<Map.Entry<String, Set<String>>>(
                            SyncProcess.PERSON_ASSOCIATION.getTitle(zone),
                            ChainingUserRegistrySynchronizer.this.transactionService.getRetryingTransactionHelper(),
                            this.personParentAssocsToDelete.entrySet(),
                            ChainingUserRegistrySynchronizer.this.workerThreads, 20,
                            ChainingUserRegistrySynchronizer.this.applicationEventPublisher,
                            ChainingUserRegistrySynchronizer.logger,
                            ChainingUserRegistrySynchronizer.this.loggingInterval);
                    groupCreator.process(new BaseBatchProcessWorker<Map.Entry<String, Set<String>>>() {
                        public String getIdentifier(Map.Entry<String, Set<String>> entry)
                        {
                            return entry.getKey() + " " + entry.getValue();
                        }

                        public void process(Map.Entry<String, Set<String>> entry) throws Throwable
                        {
                            maintainAssociationDeletions(entry.getKey());
                            maintainAssociationCreations(entry.getKey());
                            maintainAssociationCreationsToRezone(entry.getKey());
                        }
                    }, splitTxns);
                }
            }

            private void maintainAssociationDeletions(String authorityName)
            {
                boolean isPerson = AuthorityType.getAuthorityType(authorityName) == AuthorityType.USER;
                Set<String> parentsToDelete = isPerson ? this.personParentAssocsToDelete.get(authorityName)
                        : this.groupParentAssocsToDelete.get(authorityName);
                if (parentsToDelete != null && !parentsToDelete.isEmpty())
                {
                    for (String parent : parentsToDelete)
                    {
                        if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                        {
                            ChainingUserRegistrySynchronizer.logger
                                    .debug("Removing '"
                                            + ChainingUserRegistrySynchronizer.this.authorityService
                                                    .getShortName(authorityName)
                                            + "' from group '"
                                            + ChainingUserRegistrySynchronizer.this.authorityService
                                                    .getShortName(parent)
                                            + "'");
                        }
                        ChainingUserRegistrySynchronizer.this.authorityService.removeAuthority(parent, authorityName);
                    }
                }

            }

            private void maintainAssociationCreations(String authorityName)
            {
                boolean isPerson = AuthorityType.getAuthorityType(authorityName) == AuthorityType.USER;
                Set<String> parents = isPerson ? this.personParentAssocsToCreate.get(authorityName)
                        : this.groupParentAssocsToCreate.get(authorityName);
                if (parents != null && !parents.isEmpty())
                {
                    if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                    {
                        for (String groupName : parents)
                        {
                            ChainingUserRegistrySynchronizer.logger.debug("Adding '"
                                    + ChainingUserRegistrySynchronizer.this.authorityService
                                            .getShortName(authorityName)
                                    + "' to group '"
                                    + ChainingUserRegistrySynchronizer.this.authorityService.getShortName(groupName)
                                    + "'");
                        }
                    }
                    try
                    {
                        ChainingUserRegistrySynchronizer.this.authorityService.addAuthority(parents, authorityName);
                    }
                    catch (UnknownAuthorityException e)
                    {
                        // Let's force a transaction retry if a parent doesn't exist. It may be because we are
                        // waiting for another worker thread to create it
                        throw new ConcurrencyFailureException("Forcing batch retry for unknown authority", e);
                    }
                    catch (InvalidNodeRefException e)
                    {
                        // Another thread may have written the node, but it is not visible to this transaction
                        // See: ALF-5471: 'authorityMigration' patch can report 'Node does not exist'
                        throw new ConcurrencyFailureException("Forcing batch retry for invalid node", e);
                    }
                }
                // Remember that this person's associations have been maintained
                if (isPerson)
                {
                    synchronized (this)
                    {
                        this.personsProcessed.add(authorityName);
                    }
                }
            }

            private void maintainAssociationCreationsToRezone(String authorityName)
            {
                boolean isPerson = AuthorityType.getAuthorityType(authorityName) == AuthorityType.USER;

                List<String> authorities = isPerson ? this.personToRezone : this.groupToRezone;
                Map<String, Set<String>> parentAssocsToCreate = isPerson ? this.personParentAssocsToCreate : this.groupParentAssocsToCreate;

                if (authorities != null && !authorities.isEmpty() && parentAssocsToCreate.containsKey(authorityName))
                {
                    if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                    {
                        ChainingUserRegistrySynchronizer.logger.debug(
                                "Changing  '" + ChainingUserRegistrySynchronizer.this.authorityService.getShortName(authorityName)
                                        + "' to zone '" + zoneId + "'");
                    }
                    updateAuthorityZones(authorityName, ChainingUserRegistrySynchronizer.this.authorityService.getAuthorityZones(authorityName), zoneSet);
                }
            }
        } // end of Analyzer class

        // Run the first process the Group Analyzer
        final Analyzer groupAnalyzer = new Analyzer(lastModifiedMillis);
        int groupProcessedCount = groupProcessor.process(groupAnalyzer, splitTxns);

        groupAnalyzer.processGroups(userRegistry, isFullSync, splitTxns);

        // Process persons and their parent associations

        lastModifiedMillis = forceUpdate ? -1
                : getMostRecentUpdateTime(
                        ChainingUserRegistrySynchronizer.PERSON_LAST_MODIFIED_ATTRIBUTE, zoneId, splitTxns);
        lastModified = lastModifiedMillis == -1 ? null : new Date(lastModifiedMillis);
        if (ChainingUserRegistrySynchronizer.logger.isInfoEnabled())
        {
            if (lastModified == null)
            {
                ChainingUserRegistrySynchronizer.logger.info("Retrieving all users from user registry '" + zone + "'");
            }
            else
            {
                ChainingUserRegistrySynchronizer.logger.info("Retrieving users changed since "
                        + DateFormat.getDateTimeInstance().format(lastModified) + " from user registry '" + zone + "'");
            }
        }

        // User Creation and Association
        final BatchProcessor<NodeDescription> personProcessor = new BatchProcessor<NodeDescription>(
                SyncProcess.USER_CREATION.getTitle(zone),
                this.transactionService.getRetryingTransactionHelper(),
                userRegistry.getPersons(lastModified),
                this.workerThreads,
                10,
                this.applicationEventPublisher,
                ChainingUserRegistrySynchronizer.logger,
                this.loggingInterval);

        final UserRegistry userRegistryFinalRef = userRegistry;

        class PersonWorker extends BaseBatchProcessWorker<NodeDescription>
        {
            private long latestTime;

            public PersonWorker(final long latestTime)
            {
                this.latestTime = latestTime;
            }

            public long getLatestTime()
            {
                return this.latestTime;
            }

            public String getIdentifier(NodeDescription entry)
            {
                return entry.getSourceId();
            }

            public void process(NodeDescription person) throws Throwable
            {
                // Make a mutable copy of the person properties, since they get written back to by person service
                HashMap<QName, Serializable> personProperties = new HashMap<QName, Serializable>(person.getProperties());
                String personName = personProperties.get(ContentModel.PROP_USERNAME).toString().trim();
                personProperties.put(ContentModel.PROP_USERNAME, personName);

                if (Boolean.parseBoolean(ChainingUserRegistrySynchronizer.this.externalUserControl)
                        && ChainingUserRegistrySynchronizer.this.externalUserControlSubsystemName.equals(zone)
                        && userRegistryFinalRef instanceof LDAPUserRegistry)
                {
                    try
                    {
                        LDAPUserRegistry ldapUserRegistry = (LDAPUserRegistry) userRegistryFinalRef;

                        if (ldapUserRegistry.getUserAccountStatusInterpreter() != null)
                        {
                            QName propertyNameToCheck = QName.createQName(NamespaceService.CONTENT_MODEL_1_0_URI, "userAccountStatusProperty");

                            if (personProperties.get(propertyNameToCheck) != null || ldapUserRegistry.getUserAccountStatusInterpreter().acceptsNullArgument())
                            {
                                boolean isUserAccountDisabled = ldapUserRegistry.getUserAccountStatusInterpreter().isUserAccountDisabled(
                                        personProperties.get(propertyNameToCheck));

                                personProperties.put(ContentModel.PROP_ENABLED, !isUserAccountDisabled);
                            }
                        }
                    }
                    catch (IllegalArgumentException iae)
                    {
                        // Can be thrown by certain implementations of AbstractDirectoryServiceUserAccountStatusInterpreter;
                        // We'll just log it.
                        ChainingUserRegistrySynchronizer.logger.debug(iae.getMessage(), iae);
                    }
                }

                // for invalid names will throw ConstraintException that will be catched by BatchProcessor$TxnCallback
                nameChecker.evaluate(personName);
                Set<String> zones = ChainingUserRegistrySynchronizer.this.authorityService
                        .getAuthorityZones(personName);
                if (zones == null)
                {
                    // The person did not exist at all
                    if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                    {
                        ChainingUserRegistrySynchronizer.logger.debug("Creating user '" + personName + "'");
                    }
                    ChainingUserRegistrySynchronizer.this.personService.createPerson(personProperties, zoneSet);
                }
                else if (zones.contains(zoneId))
                {
                    // The person already existed in this zone: update the person
                    if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                    {
                        ChainingUserRegistrySynchronizer.logger.debug("Updating user '" + personName + "'");
                    }
                    ChainingUserRegistrySynchronizer.this.personService.setPersonProperties(personName,
                            personProperties, false);
                }
                else
                {
                    // Check whether the user is in any of the authentication chain zones
                    Set<String> intersection = new TreeSet<String>(zones);
                    intersection.retainAll(allZoneIds);
                    // Check whether the user is in any of the higher priority authentication chain zones
                    Set<String> visited = new TreeSet<String>(intersection);
                    visited.retainAll(visitedZoneIds);
                    if (visited.size() > 0)
                    {
                        // A person that exists in a different zone with higher precedence - ignore
                        return;
                    }

                    else if (!allowDeletions || intersection.isEmpty())
                    {
                        // The person exists, but in a different zone. Either deletions are disallowed or the zone is
                        // not in the authentication chain. May be due to upgrade or zone changes. Let's re-zone them
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                        {
                            ChainingUserRegistrySynchronizer.logger.warn("Updating user '" + personName
                                    + "'. This user will in future be assumed to originate from user registry '" + zone
                                    + "'.");
                        }
                        updateAuthorityZones(personName, zones, zoneSet);
                        ChainingUserRegistrySynchronizer.this.personService.setPersonProperties(personName,
                                personProperties, false);
                    }
                    else
                    {
                        // The person existed, but in a zone with lower precedence
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled())
                        {
                            ChainingUserRegistrySynchronizer.logger
                                    .warn("Recreating occluded user '"
                                            + personName
                                            + "'. This user was previously created through synchronization with a lower priority user registry.");
                        }
                        ChainingUserRegistrySynchronizer.this.personService.deletePerson(personName);
                        ChainingUserRegistrySynchronizer.this.personService.createPerson(personProperties, zoneSet);
                    }
                }

                // Maintain association deletions and creations in one shot (safe to do this with persons as we can't
                // create cycles)
                groupAnalyzer.maintainAssociationDeletions(personName);
                groupAnalyzer.maintainAssociationCreations(personName);
                groupAnalyzer.maintainAssociationCreationsToRezone(personName);

                synchronized (this)
                {
                    // Maintain the last modified date
                    Date personLastModified = person.getLastModified();
                    if (personLastModified != null)
                    {
                        this.latestTime = Math.max(this.latestTime, personLastModified.getTime());
                    }
                }
            }
        }

        PersonWorker persons = new PersonWorker(lastModifiedMillis);
        int personProcessedCount = personProcessor.process(persons, splitTxns);

        // Process those associations to persons who themselves have not been updated
        groupAnalyzer.finalizeAssociations(userRegistry, splitTxns);

        // Only now that the whole tree has been processed is it safe to persist the last modified dates
        long latestTime = groupAnalyzer.getLatestTime();
        if (latestTime != -1)
        {
            setMostRecentUpdateTime(ChainingUserRegistrySynchronizer.GROUP_LAST_MODIFIED_ATTRIBUTE, zoneId, latestTime,
                    splitTxns);
        }
        latestTime = persons.getLatestTime();
        if (latestTime != -1)
        {
            setMostRecentUpdateTime(ChainingUserRegistrySynchronizer.PERSON_LAST_MODIFIED_ATTRIBUTE, zoneId,
                    latestTime, splitTxns);
        }

        // Delete authorities if we have complete information for the zone
        Set<String> deletionCandidates = groupAnalyzer.getDeletionCandidates();
        if (isFullSync && allowDeletions && !deletionCandidates.isEmpty())
        {
            // Batch 7 Authority Deletion
            BatchProcessor<String> authorityDeletionProcessor = new BatchProcessor<String>(
                    SyncProcess.AUTHORITY_DELETION.getTitle(zone),
                    this.transactionService.getRetryingTransactionHelper(),
                    deletionCandidates,
                    this.workerThreads,
                    10,
                    this.applicationEventPublisher,
                    ChainingUserRegistrySynchronizer.logger,
                    this.loggingInterval);
            class AuthorityDeleter extends BaseBatchProcessWorker<String>
            {
                private int personProcessedCount;
                private int groupProcessedCount;

                public int getPersonProcessedCount()
                {
                    return this.personProcessedCount;
                }

                public int getGroupProcessedCount()
                {
                    return this.groupProcessedCount;
                }

                public String getIdentifier(String entry)
                {
                    return entry;
                }

                public void process(String authority) throws Throwable
                {
                    if (AuthorityType.getAuthorityType(authority) == AuthorityType.USER)
                    {
                        if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                        {
                            ChainingUserRegistrySynchronizer.logger.debug("Deleting user '" + authority + "'");
                        }
                        ChainingUserRegistrySynchronizer.this.personService.deletePerson(authority);
                        synchronized (this)
                        {
                            this.personProcessedCount++;
                        }
                    }
                    else
                    {
                        if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled())
                        {
                            ChainingUserRegistrySynchronizer.logger.debug("Deleting group '"
                                    + ChainingUserRegistrySynchronizer.this.authorityService.getShortName(authority)
                                    + "'");
                        }
                        ChainingUserRegistrySynchronizer.this.authorityService.deleteAuthority(authority);
                        synchronized (this)
                        {
                            this.groupProcessedCount++;
                        }
                    }
                }
            }
            AuthorityDeleter authorityDeleter = new AuthorityDeleter();
            authorityDeletionProcessor.process(authorityDeleter, splitTxns);
            groupProcessedCount += authorityDeleter.getGroupProcessedCount();
            personProcessedCount += authorityDeleter.getPersonProcessedCount();
        }

        // Remember we have visited this zone
        visitedZoneIds.add(zoneId);

        Object statusParams[] = {personProcessedCount, groupProcessedCount};
        final String statusMessage = I18NUtil.getMessage("synchronization.summary.status", statusParams);

        if (ChainingUserRegistrySynchronizer.logger.isInfoEnabled())
        {
            ChainingUserRegistrySynchronizer.logger.info("Finished synchronizing users and groups with user registry '"
                    + zone + "'");
            ChainingUserRegistrySynchronizer.logger.info(statusMessage);
        }

        notifySyncDirectoryEnd(zone, statusMessage);

    } // syncWithPlugin

    /**
     * Gets the persisted most recent update time for a label and zone.
     * 
     * @param label
     *            the label
     * @param zoneId
     *            the zone id
     * @param splitTxns
     *            split transactions, if true run this in a separate transaction
     * @return the most recent update time in milliseconds
     */
    private long getMostRecentUpdateTime(final String label, final String zoneId, boolean splitTxns)
    {
        return this.transactionService.getRetryingTransactionHelper().doInTransaction(
                new RetryingTransactionCallback<Long>() {
                    public Long execute() throws Throwable
                    {
                        Long updateTime = (Long) ChainingUserRegistrySynchronizer.this.attributeService.getAttribute(
                                ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH, label, zoneId);
                        return updateTime == null ? -1 : updateTime;
                    }
                }, true, splitTxns);
    }

    /**
     * Persists the most recent update time for a label and zone.
     * 
     * @param label
     *            the label
     * @param zoneId
     *            the zone id
     * @param lastModifiedMillis
     *            the update time in milliseconds
     * @param splitTxns
     *            Can the modifications to Alfresco be split across multiple transactions for maximum performance? If <code>true</code>, the attribute is persisted in a new transaction for increased performance and reliability.
     */
    private void setMostRecentUpdateTime(final String label, final String zoneId, final long lastModifiedMillis,
            boolean splitTxns)
    {
        this.transactionService.getRetryingTransactionHelper().doInTransaction(
                new RetryingTransactionHelper.RetryingTransactionCallback<Object>() {
                    public Object execute() throws Throwable
                    {
                        ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(Long
                                .valueOf(lastModifiedMillis), ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                                label, zoneId);
                        return null;
                    }
                }, false, splitTxns);
    }

    /**
     * Gets the default set of zones to set on a person or group belonging to the user registry with the given zone ID. We add the default zone as well as the zone corresponding to the user registry so that the users and groups are visible in the UI.
     * 
     * @param zoneId
     *            the zone id
     * @return the zone set
     */
    private Set<String> getZones(final String zoneId)
    {
        Set<String> zones = new HashSet<String>(5);
        zones.add(AuthorityService.ZONE_APP_DEFAULT);
        zones.add(zoneId);
        return zones;
    }

    /**
     * Modifies an authority's zone set from oldZones to newZones in the most efficient manner (avoiding unnecessary reindexing cost).
     */
    private void updateAuthorityZones(String authorityName, Set<String> oldZones, final Set<String> newZones)
    {
        Set<String> zonesToRemove = new HashSet<String>(oldZones);
        zonesToRemove.removeAll(newZones);
        // Let's keep the authority in the alfresco auth zone if it was already there. Otherwise we may have to
        // regenerate all paths to this authority from site groups, which could be very expensive!
        zonesToRemove.remove(AuthorityService.ZONE_AUTH_ALFRESCO);
        if (!zonesToRemove.isEmpty())
        {
            this.authorityService.removeAuthorityFromZones(authorityName, zonesToRemove);
        }
        Set<String> zonesToAdd = new HashSet<String>(newZones);
        zonesToAdd.removeAll(oldZones);
        if (!zonesToAdd.isEmpty())
        {
            // Prevents the authority from being added to zones where already is
            Set<String> currentZones = this.authorityService.getAuthorityZones(authorityName);
            if (currentZones != null && !currentZones.isEmpty())
            {
                zonesToAdd.removeAll(currentZones);
            }
            this.authorityService.addAuthorityToZones(authorityName, zonesToAdd);
        }
    }

    /**
     * Checks if the supplied authority is part of a certain zone
     *
     * @param authorityName
     *            the name of authority to check
     * @param authorityZones
     *            the zones where authority is
     * @param zoneToCheck
     *            the zone to check
     * @return true in case the authority is in supplied zone
     */
    private boolean isInZone(String authorityName, Set<String> authorityZones, String zoneToCheck)
    {
        return authorityName != null && authorityZones != null && zoneToCheck != null && authorityZones.contains(zoneToCheck);
    }

    @Override
    protected void onBootstrap(ApplicationEvent event)
    {
        // Do an initial differential sync on startup, using transaction splitting. This ensures that on the very
        // first startup, we don't have to wait for a very long login operation to trigger the first sync!
        if (this.syncOnStartup)
        {
            AuthenticationUtil.runAs(new RunAsWork<Object>() {
                public Object doWork() throws Exception
                {
                    try
                    {
                        synchronizeInternal(false, false, true);
                    }
                    catch (Exception e)
                    {
                        ChainingUserRegistrySynchronizer.logger.warn("Failed initial synchronize with user registries",
                                e);
                    }
                    return null;
                }
            }, AuthenticationUtil.getSystemUserName());
        }
    }

    /* (non-Javadoc)
     * 
     * @seeorg.springframework.extensions.surf.util.AbstractLifecycleBean#onShutdown(org.springframework.context. ApplicationEvent) */
    @Override
    protected void onShutdown(ApplicationEvent event)
    {}

    protected abstract class BaseBatchProcessWorker<T> implements BatchProcessWorker<T>
    {
        public final void beforeProcess() throws Throwable
        {
            // Authentication
            AuthenticationUtil.setRunAsUser(AuthenticationUtil.getSystemUserName());
        }

        public final void afterProcess() throws Throwable
        {
            // Clear authentication
            AuthenticationUtil.clearCurrentSecurityContext();
        }
    }

    private void notifySyncStart(final Set<String> toSync)
    {
        final String serverId = sysAdminParams.getAlfrescoHost() + ":" + sysAdminParams.getAlfrescoPort();
        this.applicationEventPublisher.publishEvent(new SynchronizeStartEvent(this));
        this.transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Void>() {
            @Override
            public Void execute() throws Throwable
            {
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        new Date().getTime(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.START_TIME_ATTRIBUTE);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        -1L,
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.END_TIME_ATTRIBUTE);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        serverId,
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.SERVER_ATTRIBUTE);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        SyncStatus.IN_PROGRESS.toString(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        null,
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.LAST_ERROR_ATTRIBUTE);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        "",
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.SUMMARY_ATTRIBUTE);

                for (String zoneId : toSync)
                {
                    ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                            SyncStatus.WAITING.toString(),
                            ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                            ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE,
                            zoneId);
                }

                return null;
            }
        }, false, true);
    }

    private void notifySyncEnd()
    {
        this.applicationEventPublisher.publishEvent(new SynchronizeEndEvent(this));
        this.transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Void>() {
            @Override
            public Void execute() throws Throwable
            {
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        SyncStatus.COMPLETE.toString(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        new Date().getTime(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.END_TIME_ATTRIBUTE);
                return null;
            }
        }, false, true);

    }

    private void notifySyncEnd(final Exception e)
    {
        this.applicationEventPublisher.publishEvent(new SynchronizeEndEvent(this, e));
        this.transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Void>() {
            @Override
            public Void execute() throws Throwable
            {
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        e.getMessage(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.LAST_ERROR_ATTRIBUTE);

                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        SyncStatus.COMPLETE_ERROR.toString(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE);

                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        new Date().getTime(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.END_TIME_ATTRIBUTE);

                return null;
            }
        }, false, true);
    }

    private void notifyZoneDeleted(final String zoneId)
    {
        // this.applicationEventPublisher.publishEvent(new SynchronizeDirectoryDeleteZoneEvent(this, zoneId, batchProcessNames));
        this.transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Void>() {
            @Override
            public Void execute() throws Throwable
            {
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        "",
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE,
                        zoneId);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        "",
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.SUMMARY_ATTRIBUTE,
                        zoneId);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        null,
                        ChainingUserRegistrySynchronizer.LAST_ERROR_ATTRIBUTE,
                        ChainingUserRegistrySynchronizer.SUMMARY_ATTRIBUTE,
                        zoneId);

                return null;
            }
        }, false, true);
    }

    private void notifySyncDirectoryStart(final String zoneId, final String[] batchProcessNames)
    {
        this.applicationEventPublisher.publishEvent(new SynchronizeDirectoryStartEvent(this, zoneId, batchProcessNames));
        this.transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Void>() {
            @Override
            public Void execute() throws Throwable
            {
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        SyncStatus.IN_PROGRESS.toString(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE,
                        zoneId);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        "",
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.SUMMARY_ATTRIBUTE,
                        zoneId);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        null,
                        ChainingUserRegistrySynchronizer.LAST_ERROR_ATTRIBUTE,
                        ChainingUserRegistrySynchronizer.SUMMARY_ATTRIBUTE,
                        zoneId);
                return null;
            }
        }, false, true);

    }

    private void notifySyncDirectoryEnd(final String zoneId, final String statusMessage)
    {
        this.applicationEventPublisher.publishEvent(new SynchronizeDirectoryEndEvent(this, zoneId));

        this.transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Void>() {
            @Override
            public Void execute() throws Throwable
            {
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        SyncStatus.COMPLETE.toString(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE,
                        zoneId);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        "",
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.LAST_ERROR_ATTRIBUTE,
                        zoneId);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        statusMessage,
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.SUMMARY_ATTRIBUTE,
                        zoneId);
                return null;
            }
        }, false, true);

    }

    private void notifySyncDirectoryEnd(final String zoneId, final Exception e)
    {
        this.applicationEventPublisher.publishEvent(new SynchronizeDirectoryEndEvent(this, zoneId, e));
        ChainingUserRegistrySynchronizer.logger.error("Synchronization aborted due to error", e);
        this.transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Void>() {
            @Override
            public Void execute() throws Throwable
            {
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        SyncStatus.COMPLETE_ERROR.toString(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE,
                        zoneId);
                ChainingUserRegistrySynchronizer.this.attributeService.setAttribute(
                        e.getMessage(),
                        ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH,
                        ChainingUserRegistrySynchronizer.LAST_ERROR_ATTRIBUTE,
                        zoneId);
                return null;
            }
        }, false, true);
    }

    @Override
    public Date getSyncStartTime()
    {
        Long start = (Long) ChainingUserRegistrySynchronizer.this.attributeService.getAttribute(
                ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH, ChainingUserRegistrySynchronizer.START_TIME_ATTRIBUTE);

        Date lastUserUpdate = start.longValue() == -1 ? null : new Date(start.longValue());
        return lastUserUpdate;
    }

    @Override
    public Date getSyncEndTime()
    {
        Long start = (Long) ChainingUserRegistrySynchronizer.this.attributeService.getAttribute(
                ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH, ChainingUserRegistrySynchronizer.END_TIME_ATTRIBUTE);

        Date lastUserUpdate = start.longValue() == -1 ? null : new Date(start.longValue());
        return lastUserUpdate;
    }

    @Override
    public String getLastErrorMessage()
    {
        String status = (String) ChainingUserRegistrySynchronizer.this.attributeService.getAttribute(
                ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH, ChainingUserRegistrySynchronizer.LAST_ERROR_ATTRIBUTE);
        return status;
    }

    @Override
    public String getLastRunOnServer()
    {
        String status = (String) ChainingUserRegistrySynchronizer.this.attributeService.getAttribute(
                ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH, ChainingUserRegistrySynchronizer.SERVER_ATTRIBUTE);
        return status;
    }

    @Override
    public String getSynchronizationStatus()
    {
        String status = (String) ChainingUserRegistrySynchronizer.this.attributeService.getAttribute(
                ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH, ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE);
        return status;

    }

    @Override
    public String getSynchronizationStatus(String zoneId)
    {
        String status = (String) ChainingUserRegistrySynchronizer.this.attributeService.getAttribute(
                ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH, ChainingUserRegistrySynchronizer.STATUS_ATTRIBUTE, zoneId);
        return status;
    }

    @Override
    public Date getSynchronizationLastUserUpdateTime(String id)
    {
        String zoneId = AuthorityService.ZONE_AUTH_EXT_PREFIX + id;
        long time = getMostRecentUpdateTime(ChainingUserRegistrySynchronizer.PERSON_LAST_MODIFIED_ATTRIBUTE, zoneId, false);
        Date lastUserUpdate = time == -1 ? null : new Date(time);
        return lastUserUpdate;
    }

    @Override
    public Date getSynchronizationLastGroupUpdateTime(String id)
    {
        String zoneId = AuthorityService.ZONE_AUTH_EXT_PREFIX + id;
        long time = getMostRecentUpdateTime(ChainingUserRegistrySynchronizer.GROUP_LAST_MODIFIED_ATTRIBUTE, zoneId, false);
        Date lastGroupUpdate = time == -1 ? null : new Date(time);
        return lastGroupUpdate;
    }

    @Override
    public String getSynchronizationLastError(String zoneId)
    {
        String status = (String) ChainingUserRegistrySynchronizer.this.attributeService.getAttribute(
                ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH, ChainingUserRegistrySynchronizer.LAST_ERROR_ATTRIBUTE, zoneId);
        return status;
    }

    @Override
    public String getSynchronizationSummary(String zoneId)
    {
        String status = (String) ChainingUserRegistrySynchronizer.this.attributeService.getAttribute(
                ChainingUserRegistrySynchronizer.ROOT_ATTRIBUTE_PATH, ChainingUserRegistrySynchronizer.SUMMARY_ATTRIBUTE, zoneId);
        return status;
    }

    public void setSysAdminParams(SysAdminParams sysAdminParams)
    {
        this.sysAdminParams = sysAdminParams;
    }

    public SysAdminParams getSysAdminParams()
    {
        return sysAdminParams;
    }

    @Override
    public void onApplicationEvent(ApplicationEvent event)
    {
        if (event instanceof AuthenticatorDeletedEvent)
        {
            AuthenticatorDeletedEvent deleteEvent = (AuthenticatorDeletedEvent) event;
            notifyZoneDeleted((String) deleteEvent.getSource());
        }
        else
        {
            // pass to the superclass
            super.onApplicationEvent(event);
        }
    }
}
